package com.hccl.answeringSystem;

import com.hankcs.hanlp.tokenizer.NLPTokenizer;
import com.hankcs.lucene.HanLPTokenizer;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.shingle.ShingleFilter;

import java.util.Set;

/**
 * Created by thinkit on 2016/3/7.
 */
public class HanLPBinaryAnalyzer extends Analyzer{
    private boolean pstemming;
    private Set<String> filter;

    public HanLPBinaryAnalyzer(Set<String> filter, boolean pstemming) {
        this.filter = filter;
    }

    public HanLPBinaryAnalyzer(boolean pstemming) {
        this.pstemming = pstemming;
    }

    public HanLPBinaryAnalyzer() {
    }

    protected Analyzer.TokenStreamComponents createComponents(String fieldName) {
        HanLPTokenizer tokenizer = new HanLPTokenizer(NLPTokenizer.SEGMENT.enableIndexMode(false).enableAllNamedEntityRecognize(false),
                this.filter, this.pstemming);
        TokenStream tokenStream = new ShingleFilter(tokenizer);
        return new Analyzer.TokenStreamComponents(tokenizer, tokenStream);
    }

}
